{
 "cells": [
  {
   "cell_type": "code",
   "execution_count": 10,
   "metadata": {},
   "outputs": [],
   "source": [
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/parsing/constituency/test-augmented.txt\n",
    "# !wget https://raw.githubusercontent.com/huseinzol05/Malay-Dataset/master/parsing/constituency/train-augmented.txt"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 11,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train-augmented.txt') as fopen:\n",
    "    train = fopen.read().split('\\n')\n",
    "    \n",
    "with open('test-augmented.txt') as fopen:\n",
    "    test = fopen.read().split('\\n')\n",
    "    \n",
    "train = list(filter(None, train))\n",
    "test = list(filter(None, test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 12,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "(3989, 442)"
      ]
     },
     "execution_count": 12,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(train), len(test)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 13,
   "metadata": {},
   "outputs": [],
   "source": [
    "train = [f'(TOP {s})' for s in train]\n",
    "test = [f'(TOP {s})' for s in test]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 14,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "['(TOP (S (PP (IN Menurut) (NP (PRP dia))) (, ,) (NP-SBJ (NN penjualan) (NN saham) (DT tersebut)) (VP (MD akan) (VP (VB berlaku) (ADJP (JJ efektif)) (SBAR (IN apabila) (S (NP-SBJ-1 (NN syarat-syarat) (SBAR (IN sebagaimana) (S (NP-SBJ-2 (-NONE- *)) (VP (VB termuat) (NP (-NONE- *-2)) (PP (IN dalam) (NP (NN perjanjian) (NN pengikatan) (NP (NN jual) (NN beli)) (NN saham))))))) (VP (MD telah) (VP (VB terpenuhi) (NP (-NONE- *-1)))))))) (. .)))',\n",
       " '(TOP (S-1 (NP-SBJ (NN Tokoh) (NN pasar)) (PRN (, ,) (SINV (VP (VB lanjut)  (S (-NONE- *T*-1))) (NP-SBJ (PRP dia)))) (VP (MD bagaimana) (VP (VB memerintah) (NP (NN keluarnya) (NP (NP (NP (NN dokumen) (NN indikator) (NN jimat)) (NNP AS)) (SBAR (IN yang) (S (NP-SBJ (-NONE- *)) (VP (MD akan) (VP (VB menunjukkan) (SBAR (-NONE- 0) (S (NP-SBJ (NP (NN pendirian) (NN jimat)) (NNP AS)) (VP (VP (VB mengelola)) (CC atau) (VP (ADVP (RB makin)) (VB lenyap)))))))))))))))',\n",
       " '(TOP (S-1 (NP-SBJ (NN Pelaku) (NN pasar)) (PRN (, ,) (SINV (VP (VB lanjut)  (S (-NONE- *T*-1))) (NP-SBJ (PRP dia)))) (VP (MD masih) (VP (VB memperdaya) (NP (NN keluarnya) (NP (NP (NP (NN dokumen) (NN indikator) (NN azimat)) (NNP AS)) (SBAR (IN yang) (S (NP-SBJ (-NONE- *)) (VP (MD akan) (VP (VB membocorkan) (SBAR (-NONE- 0) (S (NP-SBJ (NP (NN perkembangan) (NN azimat)) (NNP AS)) (VP (VP (VB melanjutkan)) (CC atau) (VP (ADVP (RB makin)) (VB meninggal)))))))))))))))',\n",
       " '(TOP (S-1 (NP-SBJ (NN Pelaku) (NN pasar)) (PRN (, ,) (SINV (VP (VB lanjut)  (S (-NONE- *T*-1))) (NP-SBJ (PRP dia)))) (VP (MD masih) (VP (VB menunggu) (NP (NN keluarnya) (NP (NP (NP (NN data) (NN indikator) (NN ekonomi)) (NNP AS)) (SBAR (IN yang) (S (NP-SBJ (-NONE- *)) (VP (MD akan) (VP (VB menunjukkan) (SBAR (-NONE- 0) (S (NP-SBJ (NP (NN pertumbuhan) (NN ekonomi)) (NNP AS)) (VP (VP (VB membaik)) (CC atau) (VP (ADVP (RB makin)) (VB melemah)))))))))))))))',\n",
       " '(TOP (S-1 (NP-SBJ (NN Pasien) (NN pasar)) (PRN (, ,) (SINV (VP (VB lanjut)  (S (-NONE- *T*-1))) (NP-SBJ (PRP kami)))) (VP (MD masih) (VP (VB menghabiskan) (NP (NN terpilihnya) (NP (NP (NP (NN dapatan) (NN skenario) (NN ekonomi)) (NNP RUSIA)) (SBAR (IN yang) (S (NP-SBJ (-NONE- *)) (VP (MD akan) (VP (VB menunjukkan) (SBAR (-NONE- 0) (S (NP-SBJ (NP (NN pertumbuhan) (NN ekonomi)) (NNP RUSIA)) (VP (VP (VB membaik)) (CC atau) (VP (ADVP (RB selalu)) (VB melemah)))))))))))))))']"
      ]
     },
     "execution_count": 14,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "train[:5]"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 15,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(train))\n",
    "    \n",
    "with open('test.txt', 'w') as fopen:\n",
    "    fopen.write('\\n'.join(test))"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 16,
   "metadata": {},
   "outputs": [],
   "source": [
    "from src.trees import *"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 17,
   "metadata": {},
   "outputs": [],
   "source": [
    "with open('train.txt') as infile:\n",
    "    treebanks = infile.read().split('\\n')"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 18,
   "metadata": {},
   "outputs": [],
   "source": [
    "trees = []\n",
    "for treebank in treebanks:\n",
    "    try:\n",
    "        tokens = treebank.replace(\"(\", \" ( \").replace(\")\", \" ) \").split()\n",
    "        def helper(index):\n",
    "            trees = []\n",
    "\n",
    "            while index < len(tokens) and tokens[index] == \"(\":\n",
    "                paren_count = 0\n",
    "                while tokens[index] == \"(\":\n",
    "                    index += 1\n",
    "                    paren_count += 1\n",
    "\n",
    "                label = tokens[index]\n",
    "                index += 1\n",
    "\n",
    "                if tokens[index] == \"(\":\n",
    "                    children, index = helper(index)\n",
    "                    trees.append(InternalTreebankNode(label, children))\n",
    "                else:\n",
    "                    word = tokens[index]\n",
    "                    index += 1\n",
    "                    trees.append(LeafTreebankNode(label, word))\n",
    "\n",
    "                while paren_count > 0:\n",
    "                    # print(tokens[index])\n",
    "                    assert tokens[index] == \")\"\n",
    "                    index += 1\n",
    "                    paren_count -= 1\n",
    "\n",
    "            return trees, index\n",
    "        ts, index = helper(0)\n",
    "        trees.extend(ts)\n",
    "    except:\n",
    "        pass"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 19,
   "metadata": {},
   "outputs": [
    {
     "data": {
      "text/plain": [
       "3912"
      ]
     },
     "execution_count": 19,
     "metadata": {},
     "output_type": "execute_result"
    }
   ],
   "source": [
    "len(trees)"
   ]
  },
  {
   "cell_type": "code",
   "execution_count": 20,
   "metadata": {},
   "outputs": [],
   "source": [
    "for i, tree in enumerate(trees):\n",
    "    if tree.label in (\"TOP\", \"ROOT\"):\n",
    "        assert len(tree.children) == 1\n",
    "        trees[i] = tree.children[0]"
   ]
  }
 ],
 "metadata": {
  "kernelspec": {
   "display_name": "Python 3",
   "language": "python",
   "name": "python3"
  },
  "language_info": {
   "codemirror_mode": {
    "name": "ipython",
    "version": 3
   },
   "file_extension": ".py",
   "mimetype": "text/x-python",
   "name": "python",
   "nbconvert_exporter": "python",
   "pygments_lexer": "ipython3",
   "version": "3.6.8"
  }
 },
 "nbformat": 4,
 "nbformat_minor": 2
}
